#!/usr/bin/env python3

import sys
import os
import pycurl, curl
import json
import time
import re

class Settings:
    #user = ''
    #password = ''
    nodes = 100000
    starting_node = 'tpope'

class GH_API:
    """
    GitHub API

    This class defines functions to access data on the githubs thru its API
    """
    api_url_nauth_pref = 'https://api.github.com'
    api_url_auth_fmt = 'https://{0}:{1}@api.github.com'

    def __init__(self, userpassword=None):
        self.g_curl = curl.Curl()
        self.g_curl.set_option(pycurl.HTTPHEADER, ['Accept: application/json', 'Connection: keep-alive'])
        self.g_curl.set_option(pycurl.FOLLOWLOCATION, 0)

        if userpassword:
            user, password = userpassword.split(':', maxsplit=1)
        else:
            user, password = None, None

        if user:
            self.url_pref = GH_API.api_url_auth_fmt.format(user, password)
        else:
            self.url_pref = GH_API.api_url_nauth_pref

    def go(self, url):
        """
        Return de-JSON-ified python object with the given url
        """
        self.g_curl.set_url(url)
        pycurl_poop = False

        try: response = self.g_curl.get()  # poop
        except pycurl.error as e: print('exception in self.g_curl.get(): {0}'.format(e.args), file=sys.stderr); pycurl_poop = True  # MULTIPLE STATEMENTS ON ONE LINE - DEAL WITH IT
        except: pass
        req_tries = 1
        while self.g_curl.info()['http-code'] not in (200, 204, 304) and req_tries < 7:
            print('retrying, code {0} for url {1}'.format(self.g_curl.info()['http-code'], url), file=sys.stderr)
            print('sleeping for {0} s'.format(2 ** (req_tries - 1)), file=sys.stderr)
            time.sleep(2 ** (req_tries - 1))
            try: response = self.g_curl.get(); pycurl_poop = False  # DEAL WITH IT
            except pycurl.error as e: print('exception in self.g_curl.get(): {0}'.format(e.args), file=sys.stderr); pycurl_poop = True  # I SAID DEAL WITH IT
            except: time.sleep(1)
            req_tries += 1
        if req_tries >= 7 or pycurl_poop:
            return None
        headers = self.g_curl.header().split('\r\n')
        answer = json.loads(response)

        for header in headers:
            # TODO: clean up this crap
            remaining_requests_m = re.match(r'^X-RateLimit-Remaining: (\d+)$', header)
            if remaining_requests_m:
                remaining_requests = int(remaining_requests_m.groups()[0])
            wake_up_time_m = re.match(r'^X-RateLimit-Reset: (\d+)$', header)
            if wake_up_time_m:
                wake_up_time = int(wake_up_time_m.groups()[0])

        # FIXME: fault-tolerance
        try:
            msg = '{0} requests remaining until {1}'.format(remaining_requests, time.strftime('%F %T', time.localtime(wake_up_time)))
            print(msg, file=sys.stderr)
        except:
            print('=' * 79, file=sys.stderr)
            print('something bad happened', file=sys.stderr)
            for header in headers:
                print(header, file=sys.stderr)
            print('=' * 79, file=sys.stderr)
            raise

        if remaining_requests is not None and remaining_requests <= 1:
            nap_time = wake_up_time - time.time() + 1
            print('Rate limit exceeded, sleeping for {0} seconds'.format(nap_time), file=sys.stderr)
            sys.stderr.flush()
            time.sleep(nap_time)

        return answer

    def user_dict(self, user):
        url = self.url_pref + '/users/' + user
        obj = self.go(url)
        if obj is None:
            print("user '{}' died before we got to know him. how sad :(".format(user), file=sys.stderr)
            obj = {'login': user, 'following': 0, 'followers': 0, 'type': '-DEAD'}
        sys.stderr.flush()
        return obj

    def _user_follow0_list(self, user, number, suffix='following'):
        url = self.url_pref + '/users/' + user + '/' + suffix
        follow0d = self.go(url)
        if type(follow0d) != type(list()):
            print('Unexpected behavior in GH_API.user_follow0_list: ' + \
                  'received object is not a list', file=sys.stderr)
            return []
        if len(follow0d) > 0:
            max_results_per_page = len(follow0d)
            tot_pages = number // max_results_per_page + 1
            page_i = 2
            while page_i <= tot_pages:
                follow0d += self.go(url + '?page={0}'.format(page_i))
                page_i += 1
        follow0 = list(map(lambda x: x['login'], follow0d))
        return follow0

    def user_followers_list(self, user, number):
        return self._user_follow0_list(user, number, 'followers')

    def user_following_list(self, user, number):
        return self._user_follow0_list(user, number, 'following')

def dl_graph(api, visited, queue):
    i = len(visited)
    node = ''
    if i > 0:
        resumed = True
    else:
        resume = False
        sys.stdout.write('[\n')
    while i < Settings.nodes:
        node = queue.pop(0)
        if node in visited:
            continue
        visited.add(node)
        n = api.user_dict(node)
        n['following_number'] = n['following']
        n['followers_number'] = n['followers']
        del n['following']
        del n['followers']

        if n['login'] != node:
            print("!!!!", file=sys.stderr)
            print("n.login != node:\nnode = " + node, file=sys.stderr)
            print(json.dumps(n, sort_keys=True, indent=4, separators=("," , ": ")), file=sys.stderr)
            sys.stderr.flush()

        n['following'] = api.user_following_list(node, n['following_number'])
        n['followers'] = api.user_followers_list(node, n['followers_number'])
        if n['type'] not in ('-DEAD',):
            for x in n['following'] + n['followers']:
                if x not in queue and x not in visited:
                    queue.append(x)

        # commit node
        #sys.stdout.write(json.dumps({node: n}, sort_keys=True, indent=2, separators=("," , ": ")))
        sys.stdout.write(json.dumps(n, sort_keys=True, indent=2, separators=("," , ": ")))
        i += 1
        if i < Settings.nodes:
            sys.stdout.write(',\n')
        else:
            sys.stdout.write('\n]\n')

        # quicksave
        if (i + 1) % 100 == 0:
            fp = open('GH_quicksave_i={0}_{1}.json'.format(i, time.strftime('%Y%m%d-%H%M%S', time.localtime())), 'w')
            json.dump([{'visited': list(visited), 'queue': queue}], fp, separators=(',', ':'))
            fp.flush()
            fp.close()
            del fp

        # TODO: memory consumption regulations

        sys.stdout.flush()
        sys.stderr.flush()

    return

def main(args):
    userpassword = None
    queue = [Settings.starting_node]
    visited_nodes = set()

    for arg in args[1:]:
        if bool(re.match(r"^resume=.*", arg)) and arg[7:] != '':
            print('resuming state..', file=sys.stderr)
            fp = open(arg[7:], 'r')
            sv = json.load(fp)
            visited_nodes = set(sv['visited'])
            queue = sv['queue']
            del sv
            fp.close()
            del fp
        if bool(re.match(r"^\w+:.*", arg)):
            userpassword = arg
        if bool(re.match(r"^start=.*", arg)) and arg[6:] != '':
            queue[0] = arg[6:]
        if bool(re.match(r"^number=.*", arg)) and arg[7:] != '':
            Settings.nodes = int(arg[7:])
    api = GH_API(userpassword)
    dl_graph(api, visited_nodes, queue)
    print('all done, exiting successfully', file=sys.stderr)

if __name__ == "__main__":
    main(sys.argv)
